#include "MySqlLexer.h"
#include "../../../parser/CharTypes.h"
#include "../../../parser/ParserException.h"
#include "../../../parser/LayoutCharacters.h"
#include "../../../parser/NotAllowCommentException.h"
#include "../../../parser/SymbolTable.h"
#include "../../../parser/Keywords.h"
#include "../../../parser/SQLParserFeature.h"
// #include <unicode/ucnv.h>
#include "../../../../utils/StringUtils.h"
#include "../../../../utils/log.h"
#include "../../../../utils/BOOL.h"
#include "../../../../DbType.h"

SymbolTable_ptr MySqlLexer::quoteTable = SymbolTable_ptr(new SymbolTable(8192));

std::shared_ptr<std::unordered_map<std::string, Token_ptr>> MySqlLexer::map_ = std::make_shared<std::unordered_map<std::string, Token_ptr>>();
std::shared_ptr<std::unordered_map<wchar_t, bool>> MySqlLexer::identifierFlags = std::make_shared<std::unordered_map<wchar_t, bool>>();
bool MySqlLexer::__init = MySqlLexer::init();
bool MySqlLexer::init()
{
  // LOG_INFO << "Keywords::DEFAULT_KEYWORDS->getKeywords().size:" << Keywords::DEFAULT_KEYWORDS->getKeywords()->size();
  MySqlLexer::map_->insert(Keywords::DEFAULT_KEYWORDS->getKeywords()->begin(), Keywords::DEFAULT_KEYWORDS->getKeywords()->end());
  MySqlLexer::map_->insert(std::make_pair("DUAL", Token_ptr(new Token(&Token::DUAL))));
  MySqlLexer::map_->insert(std::make_pair("FALSE", Token_ptr(new Token(&Token::XFALSE))));
  MySqlLexer::map_->insert(std::make_pair("IF", Token_ptr(new Token(&Token::IF))));
  MySqlLexer::map_->insert(std::make_pair("KILL", Token_ptr(new Token(&Token::KILL))));
  MySqlLexer::map_->insert(std::make_pair("LIMIT", Token_ptr(new Token(&Token::LIMIT))));
  MySqlLexer::map_->insert(std::make_pair("TRUE", Token_ptr(new Token(&Token::XTRUE))));
  MySqlLexer::map_->insert(std::make_pair("BINARY", Token_ptr(new Token(&Token::BINARY))));
  MySqlLexer::map_->insert(std::make_pair("SHOW", Token_ptr(new Token(&Token::SHOW))));
  MySqlLexer::map_->insert(std::make_pair("CACHE", Token_ptr(new Token(&Token::CACHE))));
  MySqlLexer::map_->insert(std::make_pair("ANALYZE", Token_ptr(new Token(&Token::ANALYZE))));
  MySqlLexer::map_->insert(std::make_pair("OPTIMIZE", Token_ptr(new Token(&Token::OPTIMIZE))));
  MySqlLexer::map_->insert(std::make_pair("ROW", Token_ptr(new Token(&Token::ROW))));
  MySqlLexer::map_->insert(std::make_pair("BEGIN", Token_ptr(new Token(&Token::BEGIN))));
  MySqlLexer::map_->insert(std::make_pair("END", Token_ptr(new Token(&Token::END))));
  MySqlLexer::map_->insert(std::make_pair("DIV", Token_ptr(new Token(&Token::DIV))));
  MySqlLexer::map_->insert(std::make_pair("MERGE", Token_ptr(new Token(&Token::MERGE))));
  MySqlLexer::map_->insert(std::make_pair("PARTITION", Token_ptr(new Token(&Token::PARTITION))));
  MySqlLexer::map_->insert(std::make_pair("CONTINUE", Token_ptr(new Token(&Token::CONTINUE))));
  MySqlLexer::map_->insert(std::make_pair("UNDO", Token_ptr(new Token(&Token::UNDO))));
  MySqlLexer::map_->insert(std::make_pair("SQLSTATE", Token_ptr(new Token(&Token::SQLSTATE))));
  MySqlLexer::map_->insert(std::make_pair("CONDITION", Token_ptr(new Token(&Token::CONDITION))));
  MySqlLexer::map_->insert(std::make_pair("MOD", Token_ptr(new Token(&Token::MOD))));
  MySqlLexer::map_->insert(std::make_pair("CONTAINS", Token_ptr(new Token(&Token::CONTAINS))));
  MySqlLexer::map_->insert(std::make_pair("RLIKE", Token_ptr(new Token(&Token::RLIKE))));
  MySqlLexer::map_->insert(std::make_pair("FULLTEXT", Token_ptr(new Token(&Token::FULLTEXT))));

  for (wchar_t c = 0; c < 256; ++c)
  {
    // LOG_INFO << "c:" << (uint8_t)c;
    std::string ustr;
    ustr += (char)c;

    if (c >= L'A' && c <= L'Z')
    {
      MySqlLexer::identifierFlags->insert(std::make_pair(c, true));
      // LOG_INFO << "MySqlLexer::identifierFlags[" << ustr << "]:BOOL::TRUE";
    }
    else if (c >= L'a' && c <= L'z')
    {
      MySqlLexer::identifierFlags->insert(std::make_pair(c, true));
      // LOG_INFO << "MySqlLexer::identifierFlags[" << ustr << "]:BOOL::TRUE";
    }
    else if (c >= L'0' && c <= L'9')
    {
      MySqlLexer::identifierFlags->insert(std::make_pair(c, true));
      // LOG_INFO << "MySqlLexer::identifierFlags[" << ustr << "]:BOOL::TRUE";
    }
    else
    {
      MySqlLexer::identifierFlags->insert(std::make_pair(c, false));
      // LOG_INFO << "MySqlLexer::identifierFlags[" << ustr << "]:BOOL::FALSE";
    }
  }
  // identifierFlags["`"] = BOOL::TRUE;
  // std::string tmp1 = "_";
  MySqlLexer::identifierFlags->at(L'_') = true;
  // std::string tmp2 = "$";
  MySqlLexer::identifierFlags->at(L'$') = true;
  // identifierFlags[L'-'] = BOOL::TRUE; // mysql
}

Keywords_ptr MySqlLexer::DEFAULT_MYSQL_KEYWORDS = std::shared_ptr<Keywords>(new Keywords(map_));

MySqlLexer::MySqlLexer(char *input, int inputLength, BOOL_ptr skipComment)
    : Lexer(const_cast<wchar_t *>(encode_util::UTF8ToUnicode(input).c_str()), inputLength, skipComment)
{
  this->keywords = DEFAULT_MYSQL_KEYWORDS;
  dbType = DbType_ptr(new DbType(&DbType::mysql));
}

MySqlLexer::MySqlLexer(string_ptr input) : MySqlLexer(input, BOOL::TRUE, BOOL::TRUE)
{
  dbType = DbType_ptr(new DbType(&DbType::mysql));
}

MySqlLexer::MySqlLexer(string_ptr input, SQLParserFeature_list_ptr features)
    : Lexer(encode_util::UTF8ToUnicode(*input), BOOL::TRUE)
{
  LOG_INFO << "in MySqlLexer";
  this->keepComments = BOOL::TRUE;
  this->keywords = DEFAULT_MYSQL_KEYWORDS;

  for (SQLParserFeature_ptr feature : *features)
  {
    config(feature, BOOL::TRUE);
  }
  dbType = DbType_ptr(new DbType(&DbType::mysql));
  LOG_INFO << "dbType->name:" << dbType->name->c_str();
}

MySqlLexer::MySqlLexer(string_ptr input, BOOL_ptr skipComment, BOOL_ptr keepComments)
    : Lexer(encode_util::UTF8ToUnicode(*input), skipComment)
{
  this->skipComment = skipComment;
  this->keepComments = keepComments;
  this->keywords = DEFAULT_MYSQL_KEYWORDS;
  dbType = DbType_ptr(new DbType(&DbType::mysql));
}

void MySqlLexer::scanSharp()
{
  if (ch != L'#')
  {
    std::string tmp = "illegal stat. ";
    tmp += encode_util::UnicodeToUTF8(info()).c_str();
    throw new ParserException(make_string_ptr(tmp));
  }

  if (charAt(pos_ + 1) == L'{')
  {
    scanVariable();
    return;
  }

  Token_ptr lastToken = this->token();

  scanChar();
  mark_ = pos_;
  bufPos = 0;
  for (;;)
  {
    if (ch == L'\r')
    {
      if (charAt(pos_ + 1) == L'\n')
      {
        bufPos += 2;
        scanChar();
        break;
      }
      bufPos++;
      break;
    }
    else if (ch == LayoutCharacters::EOI)
    {
      break;
    }

    if (ch == L'\n')
    {
      scanChar();
      bufPos++;
      break;
    }

    scanChar();
    bufPos++;
  }

  stringVal_ = subString(mark_ - 1, bufPos + 1);
  token_ = Token_ptr(new Token(&Token::LINE_COMMENT));
  commentCount++;
  if (keepComments)
  {
    addComment(stringVal_);
  }

  if (commentHandler != NULL && commentHandler->handle(lastToken, stringVal_))
  {
    return;
  }

  endOfComment = isEOF() ? BOOL::TRUE : BOOL::FALSE;

  if (!isAllowComment() && (isEOF() || !isSafeComment(stringVal_)))
  {
    throw new NotAllowCommentException();
  }
}

void MySqlLexer::scanVariable()
{
  if (ch != L':' && ch != L'#' && ch != L'$')
  {
    std::string tmp = "illegal variable. ";
    tmp += encode_util::UnicodeToUTF8(info()).c_str();
    throw new ParserException(make_string_ptr(tmp));
  }

  mark_ = pos_;
  bufPos = 1;

  if (charAt(pos_ + 1) == L'`')
  {
    ++pos_;
    ++bufPos;
    wchar_t ch;
    for (;;)
    {
      ch = charAt(++pos_);

      if (ch == L'`')
      {
        bufPos++;
        ch = charAt(++pos_);
        break;
      }
      else if (ch == LayoutCharacters::EOI)
      {
        std::string tmp = "illegal identifier. ";
        tmp += encode_util::UnicodeToUTF8(info()).c_str();
        throw new ParserException(make_string_ptr(tmp));
      }

      bufPos++;
      continue;
    }

    this->ch = charAt(pos_);

    stringVal_ = subString(mark_, bufPos);
    token_ = Token_ptr(new Token(&Token::VARIANT));
  }
  else if (charAt(pos_ + 1) == L'{')
  {
    ++pos_;
    ++bufPos;
    wchar_t ch;
    for (;;)
    {
      ch = charAt(++pos_);

      if (ch == L'}')
      {
        bufPos++;
        ch = charAt(++pos_);
        break;
      }
      else if (ch == LayoutCharacters::EOI)
      {
        std::string tmp = "illegal identifier. ";
        tmp += encode_util::UnicodeToUTF8(info()).c_str();
        throw new ParserException(make_string_ptr(tmp));
      }

      bufPos++;
      continue;
    }

    this->ch = charAt(pos_);

    stringVal_ = subString(mark_, bufPos);
    token_ = Token_ptr(new Token(&Token::VARIANT));
  }
  else
  {
    for (;;)
    {
      ch = charAt(++pos_);

      if (!isIdentifierChar(ch))
      {
        break;
      }

      bufPos++;
      continue;
    }
  }

  this->ch = charAt(pos_);

  stringVal_ = subString(mark_, bufPos);
  token_ = Token_ptr(new Token(&Token::VARIANT));
}

void MySqlLexer::scanVariable_at()
{
  if (ch != L'@')
  {
    std::string tmp = "illegal variable. ";
    tmp += encode_util::UnicodeToUTF8(info()).c_str();
    throw new ParserException(make_string_ptr(tmp));
  }

  mark_ = pos_;
  bufPos = 1;

  if (charAt(pos_ + 1) == L'@')
  {
    ch = charAt(++pos_);
    bufPos++;
  }

  if (charAt(pos_ + 1) == L'`')
  {
    ++pos_;
    ++bufPos;
    wchar_t ch;
    for (;;)
    {
      ch = charAt(++pos_);

      if (ch == L'`')
      {
        bufPos++;
        ++pos_;
        break;
      }
      else if (ch == LayoutCharacters::EOI)
      {
        std::string tmp = "illegal identifier. ";
        tmp += encode_util::UnicodeToUTF8(info()).c_str();
        throw new ParserException(make_string_ptr(tmp));
      }

      bufPos++;
      continue;
    }

    this->ch = charAt(pos_);

    stringVal_ = subString(mark_, bufPos);
    token_ = Token_ptr(new Token(&Token::VARIANT));
  }
  else
  {
    for (;;)
    {
      ch = charAt(++pos_);

      if (!isIdentifierChar(ch))
      {
        break;
      }

      bufPos++;
      continue;
    }
  }

  this->ch = charAt(pos_);

  stringVal_ = subString(mark_, bufPos);
  token_ = Token_ptr(new Token(&Token::VARIANT));
}

void MySqlLexer::scanIdentifier()
{
  LOG_INFO << "in scanIdentifier";
  hashLCase_ = 0;
  hash = 0;

  wchar_t first = ch;
  WLOG_INFO << L"ch:" << ch;

  if (first == L'U' &&
      isEnabled(SQLParserFeature_ptr(new SQLParserFeature(&SQLParserFeature::Presto))) &&
      charAt(pos_ + 1) == L'&' &&
      charAt(pos_ + 2) == L'\'')
  {
    initBuff(32);
    pos_ += 3;

    for (;; pos_++)
    {
      ch = charAt(pos_);
      WLOG_INFO << L"ch:" << ch << std::endl;
      if (isEOF())
      {
        Object_list_ptr tmp = std::make_shared<std::list<Object_ptr>>();
        lexError(L"unclosed.str.lit", tmp);
        return;
      }

      if (ch == L'\'')
      {
        ch = charAt(++pos_);
        WLOG_INFO << L"ch:" << ch << std::endl;
        break;
      }

      if (ch == L'\\')
      {
        wchar_t c1 = charAt(++pos_);
        wchar_t c2 = charAt(++pos_);
        wchar_t c3 = charAt(++pos_);
        wchar_t c4 = charAt(++pos_);

        std::wstring tmp;
        if (ch == L'+')
        {
          wchar_t c5 = charAt(++pos_);
          wchar_t c6 = charAt(++pos_);
          tmp += c1;
          tmp += c2;
          tmp += c3;
          tmp += c4;
          tmp += c5;
          tmp += c6;
          WLOG_INFO << L"tmp:" << tmp << std::endl;
        }
        else
        {
          tmp += c1;
          tmp += c2;
          tmp += c3;
          tmp += c4;
          WLOG_INFO << L"tmp:" << tmp << std::endl;
        }
        int intVal = std::stoi(encode_util::UnicodeToUTF8(tmp).c_str(), nullptr, 16);
        LOG_INFO << "intVal:" << intVal;
        putChar((char)intVal);
      }
      else
      {
        putChar(ch);
      }
    }
    std::wstring wbuf = buf;
    stringVal_ = wbuf.substr(0, bufPos);
    token_ = Token_ptr(new Token(&Token::LITERAL_CHARS));
    return;
  }

  if ((ch == L'b' ||
       ch == L'B') &&
      charAt(pos_ + 1) == L'\'')
  {
    int i = 2;
    int mark_ = pos_ + 2;
    for (;; ++i)
    {
      wchar_t ch = charAt(pos_ + i);
      if (ch == L'0' || ch == L'1')
      {
        continue;
      }
      else if (ch == L'\'')
      {
        bufPos += i;
        pos_ += (i + 1);
        stringVal_ = subString(mark_, i - 2);
        this->ch = charAt(pos_);
        token_ = Token_ptr(new Token(&Token::BITS));
        return;
      }
      else if (ch == LayoutCharacters::EOI)
      {
        std::string tmp = "illegal identifier. ";
        tmp += encode_util::UnicodeToUTF8(info()).c_str();
        throw new ParserException(make_string_ptr(tmp));
      }
      else
      {
        break;
      }
    }
  }

  if (ch == L'`')
  {
    mark_ = pos_;
    bufPos = 1;
    wchar_t ch;

    int startPos = pos_ + 1;
    int quoteIndex;

    hashLCase_ = 0xcbf29ce484222325L;
    hash = 0xcbf29ce484222325L;

    for (int i = startPos;; ++i)
    {
      if (i >= text.length())
      {
        std::string tmp = "illegal identifier. ";
        tmp += encode_util::UnicodeToUTF8(info()).c_str();
        throw new ParserException(make_string_ptr(tmp));
      }

      ch = charAt(i);

      if (L'`' == ch)
      {
        if (i + 1 < text.length() && L'`' == charAt(i + 1))
        {
          ++i;
        }
        else
        {
          // End of identifier.
          quoteIndex = i;
          break;
        }
      }

      hashLCase_ ^= ((ch >= L'A' && ch <= L'Z') ? (ch + 32) : ch);
      hashLCase_ *= 0x100000001b3L;

      hash ^= ch;
      hash *= 0x100000001b3L;
    }

    stringVal_ = quoteTable->addSymbol(text, pos_, quoteIndex + 1 - pos_, hash);
    // stringVal_ = text.substring(mark_, pos_);
    pos_ = quoteIndex + 1;
    this->ch = charAt(pos_);
    token_ = Token_ptr(new Token(&Token::IDENTIFIER));
  }
  else
  {
    bool firstFlag = CharTypes::isFirstIdentifierChar(first);
    LOG_INFO << "firstFlag:" << firstFlag;
    if (!firstFlag) 
    {
      std::string tmp = "illegal identifier. ";
      tmp += encode_util::UnicodeToUTF8(info()).c_str();
      throw new ParserException(make_string_ptr(tmp));
    }

    hashLCase_ = 0xcbf29ce484222325L;
    hash = 0xcbf29ce484222325L;

    hashLCase_ ^= ((ch >= L'A' && ch <= L'Z') ? (ch + 32) : ch);
    hashLCase_ *= 0x100000001b3L;

    hash ^= ch;
    hash *= 0x100000001b3L;

    mark_ = pos_;
    bufPos = 1;
    wchar_t ch_ = L'\0';
    for (;;)
    {
      ch_ = charAt(++pos_);
      WLOG_INFO << L"ch_:" << ch_ << std::endl;

      if (!isIdentifierChar(ch_))
      {
        break;
      }
      LOG_INFO;
      bufPos++;

      hashLCase_ ^= ((ch_ >= L'A' && ch_ <= L'Z') ? (ch_ + 32) : ch_);
      hashLCase_ *= 0x100000001b3L;

      hash ^= ch_;
      hash *= 0x100000001b3L;

      continue;
    }

    LOG_INFO << "pos_:" << pos_;
    this->ch = charAt(pos_);
    WLOG_INFO << L"this->ch:[" << this->ch << L"]" << std::endl;
    LOG_INFO << "bufPos:" << bufPos;
    LOG_INFO << "hashLCase_:" << hashLCase_;

    if (bufPos == 1)
    {
      token_ = Token_ptr(new Token(&Token::IDENTIFIER));
      stringVal_ = CharTypes::valueOf(first);
      if (stringVal_.empty())
      {
        stringVal_ = first;
      }
      return;
    }

    LOG_INFO << "hashLCase_:" << hashLCase_;
    // Token tok = keywords->getKeyword(hashLCase_);
    /* Token* */ token_ = keywords->getKeyword(hashLCase_);
    // if (token_ != nullptr || *token_ != Token::NO_EXISTING_TOKEN)
    if (token_ != nullptr)
    {
      LOG_INFO << "token name:" << token_->getName()->c_str();
      // token_ = &tok;
      if (*token_ == Token::IDENTIFIER)
      {
        stringVal_ = SymbolTable::global->addSymbol(text, mark_, bufPos, hash);
      }
      else
      {
        stringVal_.clear();
      }
    }
    else
    {
      LOG_INFO << "token is nullptr" ;
      token_ = Token_ptr(new Token(&Token::IDENTIFIER));
      stringVal_ = SymbolTable::global->addSymbol(text, mark_, bufPos, hash);
    }
    LOG_INFO << "token name:" << token_->getName()->c_str();
    WLOG_INFO << L"stringVal_:" << stringVal_ << std::endl;
  }
}

void MySqlLexer::scanString()
{
  scanString2();
}

void MySqlLexer::skipFirstHintsOrMultiCommentAndNextToken()
{
  int starIndex = pos_ + 2;

  for (;;)
  {
    starIndex = text.find(L"*", starIndex);
    if (starIndex == std::wstring::npos || starIndex == text.length() - 1)
    {
      token_ = Token_ptr(new Token(&Token::ERROR));
      return;
    }

    int slashIndex = starIndex + 1;
    if (charAt(slashIndex) == L'/')
    {
      pos_ = slashIndex + 1;
      ch = charAt(pos_);
      if (pos_ < text.length() - 6)
      {
        int pos_6 = pos_ + 6;
        wchar_t c0 = ch;
        wchar_t c1 = charAt(pos_ + 1);
        wchar_t c2 = charAt(pos_ + 2);
        wchar_t c3 = charAt(pos_ + 3);
        wchar_t c4 = charAt(pos_ + 4);
        wchar_t c5 = charAt(pos_ + 5);
        wchar_t c6 = charAt(pos_6);
        if (c0 == L's' && c1 == L'e' && c2 == L'l' && c3 == L'e' && c4 == L'c' && c5 == L't' && c6 == L' ')
        {
          this->comments->clear();
          reset(pos_6, L' ', Token_ptr(new Token(&Token::SELECT)));
          return;
        }

        if (c0 == L'i' && c1 == L'n' && c2 == L's' && c3 == L'e' && c4 == L'r' && c5 == L't' && c6 == L' ')
        {
          this->comments->clear();
          reset(pos_6, L' ', Token_ptr(new Token(&Token::INSERT)));
          return;
        }

        if (c0 == L'u' && c1 == L'p' && c2 == L'd' && c3 == L'a' && c4 == L't' && c5 == L'e' && c6 == L' ')
        {
          this->comments->clear();
          reset(pos_6, L' ', Token_ptr(new Token(&Token::UPDATE)));
          return;
        }

        if (c0 == L'd' && c1 == L'e' && c2 == L'l' && c3 == L'e' && c4 == L't' && c5 == L'e' && c6 == L' ')
        {
          this->comments->clear();
          reset(pos_6, L' ', Token_ptr(new Token(&Token::DELETE)));
          return;
        }

        if (c0 == L'S' && c1 == L'E' && c2 == L'L' && c3 == L'E' && c4 == L'C' && c5 == L'T' && c6 == L' ')
        {
          this->comments->clear();
          reset(pos_6, L' ', Token_ptr(new Token(&Token::SELECT)));
          return;
        }

        if (c0 == L'I' && c1 == L'N' && c2 == L'S' && c3 == L'E' && c4 == L'R' && c5 == L'T' && c6 == L' ')
        {
          this->comments->clear();
          reset(pos_6, L' ', Token_ptr(new Token(&Token::INSERT)));
          return;
        }

        if (c0 == L'U' && c1 == L'P' && c2 == L'D' && c3 == L'A' && c4 == L'T' && c5 == L'E' && c6 == L' ')
        {
          this->comments->clear();
          reset(pos_6, L' ', Token_ptr(new Token(&Token::UPDATE)));
          return;
        }

        if (c0 == L'D' && c1 == L'E' && c2 == L'L' && c3 == L'E' && c4 == L'T' && c5 == L'E' && c6 == L' ')
        {
          this->comments->clear();
          reset(pos_6, L' ', Token_ptr(new Token(&Token::DELETE)));
          return;
        }

        nextToken();
        return;
      }
      else
      {
        nextToken();
        return;
      }
    }
    starIndex++;
  }
}

void MySqlLexer::scanComment()
{
  Token_ptr lastToken = this->token();

  if (ch == L'-')
  {
    bool supportStandardComment = (features & SQLParserFeature::MySQLSupportStandardComment.mask) != 0;
    /*
     * just for tddl test case;
     * test case : MySqlSelectTest_plus_sub_comment.java
     */
    wchar_t before_1 = pos_ == 0 ? L' ' : charAt(pos_ - 1);
    wchar_t next_2 = charAt(pos_ + 2);
    if (isDigit(next_2))
    {
      scanChar();
      token_ = Token_ptr(new Token(&Token::SUB));
      return;
    }
    else if (supportStandardComment ||
             ((before_1 == L' ' ||
               (before_1 != L'-' &&
                before_1 != L'+')) &&
              (next_2 == L' ' ||
               next_2 == LayoutCharacters::EOI ||
               next_2 == L'\n')))
    {
      // it is comments
    }
    else if ((before_1 == L'-' || before_1 == L'+') && next_2 == L' ')
    {
      std::string tmp = "illegal state. ";
      tmp += encode_util::UnicodeToUTF8(info()).c_str();
      throw new ParserException(make_string_ptr(tmp));
    }
    else
    {
      if (ch == L'-')
      {
        scanChar();
        token_ = Token_ptr(new Token(&Token::SUB));
        return;
      }
      else if (ch == L'+')
      {
        scanChar();
        token_ = Token_ptr(new Token(&Token::PLUS));
        return;
      }
    }
  }
  else if (ch != L'/')
  {
    std::string tmp = "illegal state. ";
    tmp += encode_util::UnicodeToUTF8(info()).c_str();
    throw new ParserException(make_string_ptr(tmp));
  }

  mark_ = pos_;
  bufPos = 0;
  scanChar();

  // /*+ */
  if (ch == L'*')
  {
    scanChar();
    bufPos++;

    while (ch == L' ')
    {
      scanChar();
      bufPos++;
    }

    BOOL_ptr isHint = BOOL::FALSE;
    int startHintSp = bufPos + 1;
    if (ch == L'!'    //
        || ch == L'+' // oceanbase hints

        || ((ch == L'T' &&               // TDDL hint
             charAt(pos_ + 1) == L'D'    //
             && charAt(pos_ + 2) == L'D' //
             && charAt(pos_ + 3) == L'L') &&
            isEnabled(SQLParserFeature_ptr(new SQLParserFeature(&SQLParserFeature::TDDLHint)))))
    {
      isHint = BOOL::TRUE;
      scanChar();
      bufPos++;
    }

    int starIndex = pos_;

    // Dealing with nesting hint.
    int depth = 1;
    while (BOOL::TRUE)
    {
      wchar_t ch = charAt(starIndex);
      if (ch == L'/' && charAt(starIndex + 1) == L'*')
      {
        starIndex += 2;
        ch = charAt(starIndex);
        if (ch == L'!' || ch == L'+')
        {
          ++depth;
          ++starIndex;
          continue;
        }
      }
      else if (ch == L'*' && charAt(starIndex + 1) == L'/')
      {
        if (0 == --depth)
        {
          break;
        }
        starIndex += 2;
        continue;
      }

      if (ch == LayoutCharacters::EOI)
      {
        token_ = Token_ptr(new Token(&Token::ERROR));
        return;
      }
      ++starIndex;
    }

    if (isHint)
    {
      stringVal_ = this->subString(mark_ + startHintSp, starIndex - startHintSp - mark_);
      token_ = Token_ptr(new Token(&Token::HINT));
    }
    else
    {
      if (!optimizedForParameterized)
      {
        stringVal_ = this->subString(mark_, starIndex + 2 - mark_);
      }
      token_ = Token_ptr(new Token(&Token::MULTI_LINE_COMMENT));
      commentCount++;
      if (keepComments)
      {
        stringVal_ = this->subString(mark_, starIndex + 2 - mark_);
        addComment(stringVal_);
      }
    }
    pos_ = starIndex + 2;
    ch = charAt(pos_);

    endOfComment = isEOF() ? BOOL::TRUE : BOOL::FALSE;

    if (commentHandler != NULL &&
        commentHandler->handle(lastToken, stringVal_))
    {
      return;
    }

    if (!isHint && !isAllowComment() &&
        !isSafeComment(stringVal_))
    {
      throw new NotAllowCommentException();
    }

    return;
  }

  if (ch == L'!' &&
      isEnabled(SQLParserFeature_ptr(new SQLParserFeature(&SQLParserFeature::TDDLHint))))
  { // TDDL HINT
    scanChar();
    bufPos++;

    while (ch == L' ')
    {
      scanChar();
      bufPos++;
    }

    int startHintSp = bufPos + 1;

    int starIndex = pos_;

    for (;;)
    {
      starIndex = text.find(L"*", starIndex);
      if (starIndex == std::wstring::npos ||
          starIndex == text.length() - 1)
      {
        token_ = Token_ptr(new Token(&Token::ERROR));
        return;
      }
      if (charAt(starIndex + 1) == L'/')
      {
        stringVal_ = this->subString(mark_ + startHintSp, starIndex - startHintSp - mark_);
        token_ = Token_ptr(new Token(&Token::HINT));
        pos_ = starIndex + 2;
        ch = charAt(pos_);
        break;
      }
      starIndex++;
    }

    endOfComment = isEOF() ? BOOL::TRUE : BOOL::FALSE;

    if (commentHandler != NULL &&
        commentHandler->handle(lastToken, stringVal_))
    {
      return;
    }

    if (!isAllowComment() && !isSafeComment(stringVal_))
    {
      throw new NotAllowCommentException();
    }

    return;
  }

  if (ch == L'/' || ch == L'-')
  {
    scanChar();
    bufPos++;

    for (;;)
    {
      if (ch == L'\r')
      {
        if (charAt(pos_ + 1) == L'\n')
        {
          bufPos += 2;
          scanChar();
          break;
        }
        bufPos++;
        break;
      }
      else if (ch == LayoutCharacters::EOI)
      {
        break;
      }

      if (ch == L'\n')
      {
        scanChar();
        bufPos++;
        break;
      }

      scanChar();
      bufPos++;
    }

    stringVal_ = subString(mark_, bufPos);
    token_ = Token_ptr(new Token(&Token::LINE_COMMENT));
    commentCount++;
    if (keepComments)
    {
      addComment(stringVal_);
    }

    if (commentHandler != NULL && commentHandler->handle(lastToken, stringVal_))
    {
      return;
    }

    endOfComment = isEOF() ? BOOL::TRUE : BOOL::FALSE;

    if (!isAllowComment() && (isEOF() || !isSafeComment(stringVal_)))
    {
      throw new NotAllowCommentException();
    }

    return;
  }
}

bool MySqlLexer::isIdentifierChar(wchar_t c)
{
  // for (auto it : *MySqlLexer::identifierFlags)
  // {
  //   LOG_INFO << "it.first:" << it.first << ",it.second:" << it.second;
  // }

  LOG_INFO << "c:" << (int)c;
  auto isFind = MySqlLexer::identifierFlags->find(c);
  if (isFind != MySqlLexer::identifierFlags->end())
  {
    LOG_INFO << "isFind->second:" << isFind->second;
    return isFind->second;
  }
  else
  {
    return c != L'　' && c != L'，';
  }
}
